{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:37.379442Z",
     "start_time": "2017-10-26T19:17:36.284786Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "import re\n",
    "import collections\n",
    "import itertools\n",
    "import bcolz\n",
    "import pickle\n",
    "sys.path.append('../../lib')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import random\n",
    "import smart_open\n",
    "import h5py\n",
    "import csv\n",
    "import json\n",
    "import functools\n",
    "import time\n",
    "import string\n",
    "\n",
    "import datetime as dt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "import global_utils\n",
    "\n",
    "random_state_number = 967898"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:37.813983Z",
     "start_time": "2017-10-26T19:17:37.380748Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/gpu:0', '/gpu:1']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.python.client import device_lib\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth=True\n",
    "sess = tf.Session(config=config)\n",
    "get_available_gpus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:37.894971Z",
     "start_time": "2017-10-26T19:17:37.815241Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using matplotlib backend: TkAgg\n",
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/bicepjai/Programs/anaconda3/envs/dsotc-c3/lib/python3.6/site-packages/IPython/core/magics/pylab.py:160: UserWarning: pylab import has clobbered these variables: ['random']\n",
      "`%matplotlib` prevents importing * from pylab and numpy\n",
      "  \"\\n`%matplotlib` prevents importing * from pylab and numpy\"\n"
     ]
    }
   ],
   "source": [
    "%pylab\n",
    "%matplotlib inline\n",
    "%load_ext line_profiler\n",
    "%load_ext memory_profiler\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:37.899259Z",
     "start_time": "2017-10-26T19:17:37.896252Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "pd.options.display.max_columns = 999\n",
    "color = sns.color_palette()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:54.964710Z",
     "start_time": "2017-10-26T19:17:37.914327Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "store = pd.HDFStore('../../data_prep/processed/stage1/data_frames.h5')\n",
    "train_df = store['train_df']\n",
    "test_df = store['test_df']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:55.282043Z",
     "start_time": "2017-10-26T19:17:54.965943Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Class</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[fam58a]</td>\n",
       "      <td>[truncating, mutations]</td>\n",
       "      <td>1</td>\n",
       "      <td>[[cyclin-dependent, kinases, , cdks, , regulat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[w802*]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[q249e]</td>\n",
       "      <td>2</td>\n",
       "      <td>[[abstract, background, non-small, cell, lung,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[n454d]</td>\n",
       "      <td>3</td>\n",
       "      <td>[[recent, evidence, has, demonstrated, that, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[cbl]</td>\n",
       "      <td>[l399v]</td>\n",
       "      <td>4</td>\n",
       "      <td>[[oncogenic, mutations, in, the, monomeric, ca...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID      Gene                Variation  Class  \\\n",
       "0   0  [fam58a]  [truncating, mutations]      1   \n",
       "1   1     [cbl]                  [w802*]      2   \n",
       "2   2     [cbl]                  [q249e]      2   \n",
       "3   3     [cbl]                  [n454d]      3   \n",
       "4   4     [cbl]                  [l399v]      4   \n",
       "\n",
       "                                           Sentences  \n",
       "0  [[cyclin-dependent, kinases, , cdks, , regulat...  \n",
       "1  [[abstract, background, non-small, cell, lung,...  \n",
       "2  [[abstract, background, non-small, cell, lung,...  \n",
       "3  [[recent, evidence, has, demonstrated, that, a...  \n",
       "4  [[oncogenic, mutations, in, the, monomeric, ca...  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Gene</th>\n",
       "      <th>Variation</th>\n",
       "      <th>Sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[acsl4]</td>\n",
       "      <td>[r570s]</td>\n",
       "      <td>[[2, this, mutation, resulted, in, a, myelopro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[naglu]</td>\n",
       "      <td>[p521l]</td>\n",
       "      <td>[[abstract, the, large, tumor, suppressor, 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[pah]</td>\n",
       "      <td>[l333f]</td>\n",
       "      <td>[[vascular, endothelial, growth, factor, recep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>[ing1]</td>\n",
       "      <td>[a148d]</td>\n",
       "      <td>[[inflammatory, myofibroblastic, tumor, , imt,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>[tmem216]</td>\n",
       "      <td>[g77a]</td>\n",
       "      <td>[[abstract, retinoblastoma, is, a, pediatric, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID       Gene Variation                                          Sentences\n",
       "0   0    [acsl4]   [r570s]  [[2, this, mutation, resulted, in, a, myelopro...\n",
       "1   1    [naglu]   [p521l]  [[abstract, the, large, tumor, suppressor, 1, ...\n",
       "2   2      [pah]   [l333f]  [[vascular, endothelial, growth, factor, recep...\n",
       "3   3     [ing1]   [a148d]  [[inflammatory, myofibroblastic, tumor, , imt,...\n",
       "4   4  [tmem216]    [g77a]  [[abstract, retinoblastoma, is, a, pediatric, ..."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(train_df.head())\n",
    "display(test_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:55.379112Z",
     "start_time": "2017-10-26T19:17:55.283217Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "352220 352220\n"
     ]
    }
   ],
   "source": [
    "corpus_vocab_list, corpus_vocab_wordidx = None, None\n",
    "with open('../../data_prep/processed/stage1/vocab_words_wordidx.pkl', 'rb') as f:\n",
    "    (corpus_vocab_list, corpus_wordidx) = pickle.load(f)\n",
    "print(len(corpus_vocab_list), len(corpus_wordidx))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-14T08:20:17.449244Z",
     "start_time": "2017-08-14T08:20:15.593136Z"
    },
    "collapsed": true
   },
   "source": [
    "# Data Prep"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To control the vocabulary pass in updated corpus_wordidx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:17:55.416950Z",
     "start_time": "2017-10-26T19:17:55.380302Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5)\n",
      "(333, 5)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train_df, x_val_df = train_test_split(train_df,\n",
    "                                         test_size=0.10, random_state=random_state_number,\n",
    "                                         stratify=train_df.Class)\n",
    "\n",
    "print(x_train_df.shape)\n",
    "print(x_val_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:02.384527Z",
     "start_time": "2017-10-26T19:17:55.418059Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.contrib.keras.python.keras.utils import np_utils\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:02.388033Z",
     "start_time": "2017-10-26T19:18:02.386016Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_size=len(corpus_vocab_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## T:sent_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:14:20.659301Z",
     "start_time": "2017-10-26T00:14:20.653375Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"sentences\",\n",
    "         \"divide_document\": \"multiple_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:16:51.148007Z",
     "start_time": "2017-10-26T00:14:22.265259Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_21_T, x_train_21_G, x_train_21_V, x_train_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:16:55.113872Z",
     "start_time": "2017-10-26T00:16:51.149180Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "(2622081,) [364606, 113692, 197002, 330024, 326252, 151042, 75648, 1818, 276247, 61043, 228115, 326252, 74974, 301275, 76659, 326252, 361104, 329709, 253643, 205596, 153283, 326252, 80594, 326252, 113692, 18820, 349251, 59442, 123801, 228752, 245229, 307200, 17105, 60555, 69032, 1818, 274163, 151942, 246684, 222367, 253643, 243777, 274163, 50915, 274163, 12413, 1818, 228752, 364603, 232434, 214275, 235155, 163151, 123801, 101614, 101366, 364607]\n",
      "(2622081, 3) [364606, 97957, 364607]\n",
      "(2622081,) [364606, 326252, 364607]\n",
      "(2622081,) 6\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(np.array(x_train_21_T).shape, x_train_21_T[0])\n",
    "print(np.array(x_train_21_G).shape, x_train_21_G[0])\n",
    "print(np.array(x_train_21_V).shape, x_train_21_V[0])\n",
    "print(np.array(x_train_21_C).shape, x_train_21_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.385302Z",
     "start_time": "2017-10-26T00:16:55.114982Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_21_T, x_val_21_G, x_val_21_V, x_val_21_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.853790Z",
     "start_time": "2017-10-26T00:17:10.386619Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (293702,)\n",
      "gene (293702, 3) [364606, 112978, 364607]\n",
      "variation (293702,) [364606, 295010, 364607]\n",
      "classes (293702,) 2\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_21_T).shape)\n",
    "print(\"gene\",np.array(x_val_21_G).shape, x_val_21_G[0])\n",
    "print(\"variation\",np.array(x_val_21_V).shape, x_val_21_V[0])\n",
    "print(\"classes\",np.array(x_val_21_C).shape, x_val_21_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.857143Z",
     "start_time": "2017-10-26T00:17:10.854958Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:10.872469Z",
     "start_time": "2017-10-26T00:17:10.858162Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "MAX_SENT_LEN = 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.327704Z",
     "start_time": "2017-10-26T00:17:10.873609Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2622081, 60) (293702, 60)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_T = pad_sequences(x_train_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_21_T = pad_sequences(x_val_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_21_T.shape, x_val_21_T.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.564842Z",
     "start_time": "2017-10-26T00:17:27.328915Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "x_train_21_C = np.array(x_train_21_C) - 1\n",
    "x_val_21_C = np.array(x_val_21_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T00:17:27.631996Z",
     "start_time": "2017-10-26T00:17:27.566066Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2622081, 9) (293702, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_21_C = np_utils.to_categorical(np.array(x_train_21_C), 9)\n",
    "x_val_21_C = np_utils.to_categorical(np.array(x_val_21_C), 9)\n",
    "print(x_train_21_C.shape, x_val_21_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## T:text_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:02.408307Z",
     "start_time": "2017-10-26T19:18:02.389103Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "custom_unit_dict = {\n",
    "         \"gene_unit\"      : \"words\",\n",
    "         \"variation_unit\" : \"words\",\n",
    "         # text transformed to sentences attribute\n",
    "         \"doc_unit\"       : \"words\",\n",
    "         \"doc_form\"       : \"text\",\n",
    "         \"divide_document\": \"single_unit\"\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:12.670735Z",
     "start_time": "2017-10-26T19:18:02.409467Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)\n",
    "x_train_22_T, x_train_22_G, x_train_22_V, x_train_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:13.694335Z",
     "start_time": "2017-10-26T19:18:12.672255Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data\n",
      "text (2988,)\n",
      "gene (2988, 3) [352216, 164788, 352217]\n",
      "variation (2988,) [352216, 86196, 352217]\n",
      "classes (2988,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Train data\")\n",
    "print(\"text\",np.array(x_train_22_T).shape)\n",
    "print(\"gene\",np.array(x_train_22_G).shape, x_train_22_G[0])\n",
    "print(\"variation\",np.array(x_train_22_V).shape, x_train_22_V[0])\n",
    "print(\"classes\",np.array(x_train_22_C).shape, x_train_22_C[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:14.952504Z",
     "start_time": "2017-10-26T19:18:13.695426Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)\n",
    "x_val_22_T, x_val_22_G, x_val_22_V, x_val_22_C = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                             has_class=True,\n",
    "                                                                             add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:15.163573Z",
     "start_time": "2017-10-26T19:18:14.955384Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val data\n",
      "text (333,)\n",
      "gene (333, 3) [352216, 217983, 352217]\n",
      "variation (333,) [352216, 41934, 352217]\n",
      "classes (333,) 4\n"
     ]
    }
   ],
   "source": [
    "print(\"Val data\")\n",
    "print(\"text\",np.array(x_val_22_T).shape)\n",
    "print(\"gene\",np.array(x_val_22_G).shape, x_val_22_G[0])\n",
    "print(\"variation\",np.array(x_val_22_V).shape, x_val_22_V[0])\n",
    "print(\"classes\",np.array(x_val_22_C).shape, x_val_22_C[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### format data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:15.194305Z",
     "start_time": "2017-10-26T19:18:15.165582Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_unknown_tag_idx   = corpus_wordidx[\"<UNK>\"]\n",
    "char_unknown_tag_idx   = global_utils.char_unknown_tag_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:15.216547Z",
     "start_time": "2017-10-26T19:18:15.196051Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MAX_TEXT_LEN = 5000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:16.324521Z",
     "start_time": "2017-10-26T19:18:15.218369Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 5000) (333, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_T = pad_sequences(x_train_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "x_val_22_T = pad_sequences(x_val_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_train_22_T.shape, x_val_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:16.359304Z",
     "start_time": "2017-10-26T19:18:16.326390Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 1) (2988, 4)\n",
      "(333, 1) (333, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_train_22_G = pad_sequences(x_train_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_train_22_V = pad_sequences(x_train_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "x_val_22_G = pad_sequences(x_val_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_val_22_V = pad_sequences(x_val_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_train_22_G.shape, x_train_22_V.shape)\n",
    "print(x_val_22_G.shape, x_val_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "keras np_utils.to_categorical expects zero index categorical variables\n",
    "\n",
    "https://github.com/fchollet/keras/issues/570"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:16.369772Z",
     "start_time": "2017-10-26T19:18:16.361569Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x_train_22_C = np.array(x_train_22_C) - 1\n",
    "x_val_22_C = np.array(x_val_22_C) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:16.403750Z",
     "start_time": "2017-10-26T19:18:16.371830Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2988, 9) (333, 9)\n"
     ]
    }
   ],
   "source": [
    "x_train_22_C = np_utils.to_categorical(np.array(x_train_22_C), 9)\n",
    "x_val_22_C = np_utils.to_categorical(np.array(x_val_22_C), 9)\n",
    "print(x_train_22_C.shape, x_val_22_C.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### test Data setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:32.887420Z",
     "start_time": "2017-09-26T03:43:29.372697Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "gen_data = global_utils.GenerateDataset(test_df, corpus_wordidx)\n",
    "x_test_22_T, x_test_22_G, x_test_22_V, _ = gen_data.generate_data(custom_unit_dict, \n",
    "                                                                has_class=False,\n",
    "                                                                add_start_end_tag=True)\n",
    "del gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.178763Z",
     "start_time": "2017-09-26T03:43:32.888877Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test data\n",
      "text (986,)\n",
      "gene (986, 3) [364606, 188717, 364607]\n",
      "variation (986,) [364606, 317947, 364607]\n"
     ]
    }
   ],
   "source": [
    "print(\"Test data\")\n",
    "print(\"text\",np.array(x_test_22_T).shape)\n",
    "print(\"gene\",np.array(x_test_22_G).shape, x_test_22_G[0])\n",
    "print(\"variation\",np.array(x_test_22_V).shape, x_test_22_V[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.546461Z",
     "start_time": "2017-09-26T03:43:33.181689Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 5000)\n"
     ]
    }
   ],
   "source": [
    "x_test_22_T = pad_sequences(x_test_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,\n",
    "                                  padding=\"post\",truncating=\"post\")\n",
    "print(x_test_22_T.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-26T03:43:33.575305Z",
     "start_time": "2017-09-26T03:43:33.548386Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(986, 1) (986, 4)\n"
     ]
    }
   ],
   "source": [
    "MAX_GENE_LEN = 1\n",
    "MAX_VAR_LEN = 4\n",
    "x_test_22_G = pad_sequences(x_test_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)\n",
    "x_test_22_V = pad_sequences(x_test_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)\n",
    "\n",
    "print(x_test_22_G.shape, x_test_22_V.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedding layer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:16.422895Z",
     "start_time": "2017-10-26T19:18:16.405540Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "WORD_EMB_SIZE = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:39.169690Z",
     "start_time": "2017-10-26T19:18:16.424738Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(352220, 200)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%autoreload\n",
    "import global_utils\n",
    "ft_file_path = \"/home/bicepjai/Projects/Deep-Survey-Text-Classification/data_prep/processed/stage1/pretrained_word_vectors/ft_sg_200d_50e.vec\"\n",
    "trained_embeddings = global_utils.get_embeddings_from_ft(ft_file_path, WORD_EMB_SIZE, corpus_vocab_list)\n",
    "trained_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "### for characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:04:28.473180Z",
     "start_time": "2017-09-25T06:04:28.470711Z"
    },
    "collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CHAR_EMB_SIZE = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-09-25T06:04:28.479331Z",
     "start_time": "2017-09-25T06:04:28.474704Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75, 100)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_embeddings = np.random.randn(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE)\n",
    "char_embeddings.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:39.189732Z",
     "start_time": "2017-10-26T19:18:39.171085Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%autoreload\n",
    "from utils import MedCNN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-08-24T06:58:17.661183Z",
     "start_time": "2017-08-24T06:58:17.655020Z"
    }
   },
   "source": [
    "## model_1: paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T19:18:39.484678Z",
     "start_time": "2017-10-26T19:18:39.191011Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = MedCNN(n_cnn2_pool_pair_layers=2, fc_layer_len=128, n_filters=256, kernel_size=5,\n",
    "               dropout_porb=0.5, input_sentence_len=MAX_TEXT_LEN, \n",
    "               output_label_size=9, word_vectors=trained_embeddings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-10-26T20:22:49.647963Z",
     "start_time": "2017-10-26T19:18:39.486040Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing logs to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ed437f90d6d4c42a71f32528ff45174",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-94\n",
      "\n",
      "epoch 0 train_loss:4949.01986167 train_accuracy:0.169326241188 test_loss:2.197224617 test_accuracy:0.170236014507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a075aef7be242108b40ae9aa9d74d63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-188\n",
      "\n",
      "epoch 1 train_loss:2.197224617 train_accuracy:0.17098847523 test_loss:2.197224617 test_accuracy:0.161931818182\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b41af19320324d5a8bbe1118c2887694",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-282\n",
      "\n",
      "epoch 2 train_loss:2.197224617 train_accuracy:0.17098847523 test_loss:2.197224617 test_accuracy:0.170236014507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0821d5a1baf4b6b8c31a15bea7bc336",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-376\n",
      "\n",
      "epoch 3 train_loss:2.197224617 train_accuracy:0.17043439719 test_loss:2.197224617 test_accuracy:0.186844408512\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ea7fc99ed8341508489f9a1baa176df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-470\n",
      "\n",
      "epoch 4 train_loss:2.197224617 train_accuracy:0.171542553191 test_loss:2.197224617 test_accuracy:0.170236014507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b61aabd13a04754ab57b7326fb275b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-564\n",
      "\n",
      "epoch 5 train_loss:2.197224617 train_accuracy:0.17043439719 test_loss:2.197224617 test_accuracy:0.166083916344\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35bf854704f2432d939fa8afeeac6900",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-658\n",
      "\n",
      "epoch 6 train_loss:2.197224617 train_accuracy:0.17098847523 test_loss:2.197224617 test_accuracy:0.170236014507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4b51113339d4715b391f5626ef2dae1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-752\n",
      "\n",
      "epoch 7 train_loss:2.197224617 train_accuracy:0.17098847523 test_loss:2.197224617 test_accuracy:0.166083916344\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9782c0f2bf9469ba8c17a443dc64b33",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-846\n",
      "\n",
      "epoch 8 train_loss:2.197224617 train_accuracy:0.172096631311 test_loss:2.197224617 test_accuracy:0.174388113347\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9074d7eef4ba4a25847000799aabc6e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-940\n",
      "\n",
      "epoch 9 train_loss:2.197224617 train_accuracy:0.17098847523 test_loss:2.197224617 test_accuracy:0.178540210832\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49b768f029a74d04aab90db1a3c60a8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-1034\n",
      "\n",
      "epoch 10 train_loss:2.197224617 train_accuracy:0.17098847523 test_loss:2.197224617 test_accuracy:0.170236014507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5b79ad511654c4d921e89021252b4e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-1128\n",
      "\n",
      "epoch 11 train_loss:2.197224617 train_accuracy:0.169880319149 test_loss:2.197224617 test_accuracy:0.170236014507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc0364829e5c45ac97e5bb45520a5ca3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-1222\n",
      "\n",
      "epoch 12 train_loss:2.197224617 train_accuracy:0.17098847523 test_loss:2.197224617 test_accuracy:0.182692311027\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2f73dec3f354d86b7ffed32307368d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-1316\n",
      "\n",
      "epoch 13 train_loss:2.197224617 train_accuracy:0.169880319149 test_loss:2.197224617 test_accuracy:0.174388113347\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62e0ea9308e04eae87bb4d769a3c84f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-1410\n",
      "\n",
      "epoch 14 train_loss:2.197224617 train_accuracy:0.169880319149 test_loss:2.197224617 test_accuracy:0.170236014507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "297d1549f73c4d5fbfc56284b6dcddcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saved model checkpoint to /home/bicepjai/Projects/Deep-Survey-Text-Classification/deep_models/paper_3_medical_cnn/runs/1509045519/checkpoints/model-1504\n",
      "\n",
      "epoch 15 train_loss:2.197224617 train_accuracy:0.171542553191 test_loss:2.197224617 test_accuracy:0.166083916344\n"
     ]
    }
   ],
   "source": [
    "model.train((x_train_22_T, x_train_22_C),\n",
    "            (x_val_22_T, x_val_22_C),\n",
    "            num_epochs=16,\n",
    "            checkpoint_every_n_epoch=1,\n",
    "            evaluate_every_n_epoch=1,\n",
    "            learning_rate=0.01, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {
    "height": "913px",
    "left": "0px",
    "right": "1192px",
    "top": "52px",
    "width": "300px"
   },
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
